# -*- coding: utf-8 -*-
# import sys
# reload(sys)
# sys.setdefaultencoding("utf-8")
import multiprocessing
import time
from spider import SouGouWeiXinSpider
from conifig import connect
from Logs.log import log1


def run():
    sql = """
        select domain, classify from helper_information_net where classify < 9
    """
    cursor = connect.cursor()
    cursor.execute(sql)
    names = cursor.fetchall()  # ((domain, classify), (), ())
    num = 20
    ns0 = names[:num]
    ns1 = names[num:num*2]
    ns2 = names[num*2:num*3]
    ns3 = names[num*3:num*4]
    ns4 = names[num*4:num*5]
    ns5 = names[num*5:]
    sg = SouGouWeiXinSpider()
    process = []
    p0 = multiprocessing.Process(target=sg.start_requests, args=(ns0, 0))
    p1 = multiprocessing.Process(target=sg.start_requests, args=(ns1, 3))
    p2 = multiprocessing.Process(target=sg.start_requests, args=(ns2, 6))
    p3 = multiprocessing.Process(target=sg.start_requests, args=(ns3, 9))
    p4 = multiprocessing.Process(target=sg.start_requests, args=(ns4, 12))
    p5 = multiprocessing.Process(target=sg.start_requests, args=(ns5, 15))
    process.append(p0)
    process.append(p1)
    process.append(p2)
    process.append(p3)
    process.append(p4)
    process.append(p5)
    for p in process:
        p.start()
    for p in process:
        p.join()
    connect.close()
    cursor.close()


if __name__ == '__main__':
    start = time.time()
    log1.info("\n ----------------------------------开始爬取----------------------------------")
    run()
    end = time.time()
    log1.info("抓取完毕, 共耗时:{}s".format(end-start))
